chore: upgrade to iceberg 1.7.0 (#48771)

airbytehq · Dec 3, 2024 · f234706 · f234706
1 parent 15291b3
commit f234706
Show file tree

Hide file tree

Showing 5 changed files with 19 additions and 17 deletions.
diff --git a/airbyte-cdk/bulk/toolkits/load-iceberg-parquet/build.gradle b/airbyte-cdk/bulk/toolkits/load-iceberg-parquet/build.gradle
@@ -1,12 +1,16 @@
+ext {
+    apacheIcebergVersion = '1.7.0'
+}
+
 dependencies {
     api project(':airbyte-cdk:bulk:toolkits:bulk-cdk-toolkit-load-s3')
 
     implementation project(':airbyte-cdk:bulk:core:bulk-cdk-core-base')
     implementation project(':airbyte-cdk:bulk:core:bulk-cdk-core-load')
-    api 'org.apache.iceberg:iceberg-core:1.6.1'
-    api 'org.apache.iceberg:iceberg-api:1.6.1'
-    api 'org.apache.iceberg:iceberg-parquet:1.6.1'
-    api 'org.apache.iceberg:iceberg-nessie:1.6.1'
+    api "org.apache.iceberg:iceberg-core:${project.ext.apacheIcebergVersion}"
+    api "org.apache.iceberg:iceberg-api:${project.ext.apacheIcebergVersion}"
+    api "org.apache.iceberg:iceberg-parquet:${project.ext.apacheIcebergVersion}"
+    api "org.apache.iceberg:iceberg-nessie:${project.ext.apacheIcebergVersion}"
 
     testFixturesImplementation testFixtures(project(":airbyte-cdk:bulk:core:bulk-cdk-core-load"))
 }
diff --git a/airbyte-integrations/connectors/destination-iceberg-v2/build.gradle b/airbyte-integrations/connectors/destination-iceberg-v2/build.gradle
@@ -19,9 +19,9 @@ application {
 }
 
 ext {
-    apacheIcebergVersion = "1.6.1"
-    awsSdkVersion = "2.29.9"
-    junitVersion = "5.11.3"
+    apacheIcebergVersion = '1.7.0'
+    awsSdkVersion = '2.29.9'
+    junitVersion = '5.11.3'
 }
 
 // Uncomment to run locally

diff --git a/airbyte-integrations/connectors/destination-iceberg-v2/metadata.yaml b/airbyte-integrations/connectors/destination-iceberg-v2/metadata.yaml
@@ -2,7 +2,7 @@ data:
   connectorSubtype: file
   connectorType: destination
   definitionId: 37a928c1-2d5c-431a-a97d-ae236bd1ea0c
-  dockerImageTag: 0.1.9
+  dockerImageTag: 0.1.10
   dockerRepository: airbyte/destination-iceberg-v2
   githubIssueLabel: destination-iceberg-v2
   icon: s3.svg

diff --git a/.../src/main/kotlin/io/airbyte/integrations/destination/iceberg/v2/io/IcebergTableCleaner.kt b/.../src/main/kotlin/io/airbyte/integrations/destination/iceberg/v2/io/IcebergTableCleaner.kt
@@ -56,14 +56,12 @@ class IcebergTableCleaner(private val icebergUtil: IcebergUtil) {
 
         table.newScan().planFiles().use { tasks ->
             tasks
-                .filter { task ->
-                    genIdsToDelete.any { id -> task.file().path().toString().contains(id) }
-                }
+                .filter { task -> genIdsToDelete.any { id -> task.file().location().contains(id) } }
                 .forEach { task ->
                     table
                         .newDelete()
                         .toBranch(stagingBranchName)
-                        .deleteFile(task.file().path())
+                        .deleteFile(task.file().location())
                         .commit()
                 }
         }

diff --git a/.../test/kotlin/io/airbyte/integrations/destination/iceberg/v2/io/IcebergTableCleanerTest.kt b/.../test/kotlin/io/airbyte/integrations/destination/iceberg/v2/io/IcebergTableCleanerTest.kt
@@ -102,11 +102,11 @@ internal class IcebergTableCleanerTest {
         every { tasks.close() } just Runs
         every { table.newScan().planFiles() } returns tasks
 
-        every { fileScanTask.file().path().toString() } returns filePathToDelete
+        every { fileScanTask.file().location() } returns filePathToDelete
 
         val delete = mockk<DeleteFiles>()
         every { table.newDelete().toBranch("staging") } returns delete
-        every { delete.deleteFile(fileScanTask.file().path()) } returns delete
+        every { delete.deleteFile(filePathToDelete) } returns delete
         every { delete.commit() } just Runs
 
         assertDoesNotThrow {
@@ -116,7 +116,7 @@ internal class IcebergTableCleanerTest {
         verify {
             icebergUtil.assertGenerationIdSuffixIsOfValidFormat(generationIdSuffix)
             table.newDelete().toBranch(eq("staging"))
-            delete.deleteFile(fileScanTask.file().path())
+            delete.deleteFile(filePathToDelete)
             delete.commit()
         }
     }
@@ -138,11 +138,11 @@ internal class IcebergTableCleanerTest {
         every { tasks.close() } just Runs
         every { table.newScan().planFiles() } returns tasks
 
-        every { fileScanTask.file().path().toString() } returns filePathToDelete
+        every { fileScanTask.file().location().toString() } returns filePathToDelete
 
         val delete = mockk<DeleteFiles>()
         every { table.newDelete().toBranch("staging") } returns delete
-        every { delete.deleteFile(fileScanTask.file().path()) } returns delete
+        every { delete.deleteFile(fileScanTask.file().location()) } returns delete
         every { delete.commit() } just Runs
 
         assertDoesNotThrow {